package services;

import java.io.BufferedReader;
import java.io.IOException;
import java.io.InputStreamReader;
import java.io.OutputStreamWriter;
import java.io.UnsupportedEncodingException;
import java.net.HttpURLConnection;
import java.net.MalformedURLException;
import java.net.URL;
import java.net.URLEncoder;
import java.nio.charset.Charset;
import java.util.ArrayList;
import java.util.Date;
import java.util.List;
import java.util.Random;
import java.util.regex.Matcher;
import java.util.regex.Pattern;

import org.jsoup.Jsoup;
import org.jsoup.nodes.Document;
import org.jsoup.nodes.Element;
import org.jsoup.select.Elements;

import com.googlecode.objectify.Objectify;
import com.googlecode.objectify.ObjectifyService;
import com.googlecode.objectify.Query;

import debug.DomainDebug;

import models.Domain;
import models.RecentSite;

public class UpdateDomain {
	public UpdateDomain(String domainName) {
		Domain domain = new Domain();
		domain.setName(domainName);
		domain = getSiteInfo(domain);
		domain = getAlexa(domain);
		domain = getGooglePageRank(domain);
		domain = getAgeDomain(domain);
		domain = getWhois(domain);
		domain = getIp(domain);
		domain = getCompete(domain);
		domain = getRandomSites(domain);
		domain = getNearSites(domain);
		
		//Update ip
		if (domain.getIpAddress() != null && !domain.getIpAddress().isEmpty()) {
			new UpdateIp(domain.getIpAddress(), domainName);
		}
		
		//Update googleAdsnse
		if (domain.getGoogleAdsense() != null && !domain.getGoogleAdsense().isEmpty()) {
			new UpdateGoogleAdsense(domain.getGoogleAdsense(), domainName);
		}
		
		//Update googleAdsnse
		if (domain.getGoogleAnalytic() != null && !domain.getGoogleAnalytic().isEmpty()) {
			new UpdateGoogleAnalytic(domain.getGoogleAnalytic(), domainName);
		}
		
		//Save data
		try {
			ObjectifyService.register(Domain.class);
		} catch (Exception e) {
			// TODO: handle exception
		}
		
		Objectify ofy = ObjectifyService.begin();
		ofy.put(domain);
		
		//Set recentSiteUpdate
		RecentSite.setRecentUpdate(domain);
		
		System.out.println("Update domain success: " + domainName);
	}
	
	public static Domain getSiteInfo(Domain domain)
	{
		String domainName = domain.getName();
		try {
			
			String url = "http://" + domainName;
			URL dataURL = new URL(url);
			
			String charset   = "UTF-8";
			
		    HttpURLConnection connection = (HttpURLConnection) dataURL.openConnection();
		    connection.setInstanceFollowRedirects(true);

		    BufferedReader reader = new BufferedReader(new InputStreamReader(connection.getInputStream(), Charset.forName(charset)));
			String content = "";
			connection.getInputStream().mark(20000);
			while(1==1)
			{
				String str = reader.readLine();
				if(str==null) break;
				content+=str;
			}
			
			String server = "";
			try {
				server = connection.getHeaderField("Server");
			} catch (Exception e) {
				
			}
			String powerBy = "";
			try {
				powerBy = connection.getHeaderField("X-Powered-By");
			} catch (Exception e) {
				// TODO: handle exception
			}
			
			Document doc = null;
			try {
				doc = Jsoup.connect(url).get();
			} catch (Exception e) {
				url = "http://www." + domainName;
				doc = Jsoup.connect(url).get();
			}
			
			String title = "";
			String description = "";
			String keywords  = "";
			
			try
			{
				try {
					title = doc.select("title").first().text();
				} catch (Exception e) {
					e.printStackTrace();
				}

				try {
					keywords = doc.select("meta[name=keywords]").first().attr("content");
				} catch (Exception e) {
					try {
						keywords = doc.select("meta[http-equiv=keywords]").first().attr("content");
					} catch (Exception e2) {
						//e.printStackTrace();
					}		
				}
				
				try {
					description = doc.select("meta[name=description]").first().attr("content");
				} catch (Exception e) {
					try {
						description = doc.select("meta[http-equiv=description]").first().attr("content");
					} catch (Exception e2) {
						if(doc.select("body").first().text().length() > 260)
						{
							description = doc.select("body").first().text().substring(0,250).replaceAll("[ ]+[\\w\\W]{0,10}+$", "")+" ...";
						} else {
							description = doc.select("body").first().text();
						}
					}
				}
				if(description.length()>260)
				{
					description = description.substring(0,250);
				}

			}
			catch(Exception ex)
			{
				ex.printStackTrace();
				
			}
			
			int begin = content.indexOf("_gaq.push(['_setAccount', '");
			String ananyticAccount = "";
			if(begin>0)
			{
				begin+="_gaq.push(['_setAccount', '".length();
				int end =  content.indexOf("'",begin);
				ananyticAccount = content.substring(begin, end);
				begin=ananyticAccount.lastIndexOf("-")-1;
				
				ananyticAccount = ananyticAccount.substring(0, begin);
				System.out.println("ananyticAccount :"+ananyticAccount);
			
			}
			else
			{
				 begin = content.indexOf("_gaq.push([\"_setAccount\", \"");
				if(begin>0)
				{
					begin+="_gaq.push([\"_setAccount\", \"".length();
					int end =  content.indexOf("\"",begin);
					ananyticAccount = content.substring(begin, end);
					System.out.println("ananyticAccount :"+ananyticAccount);
				} else {
					begin = content.indexOf("_uacct");
					if(begin > 0)
					{
						begin+="_uacct".length();
						ananyticAccount = content.substring(begin,begin+30).replaceAll("^[ =\\'\\\"]+", "").replaceAll("[^a-zA-Z0-9\\-\\_]+[\\w\\W]+", "");
						begin=ananyticAccount.lastIndexOf("-");
						
						ananyticAccount = ananyticAccount.substring(0, begin);
						
						System.out.println("ananyticAccount :"+ananyticAccount);
					} else {
						Pattern p = Pattern.compile("UA-[0-9]+");

				        Matcher m = p.matcher(content);
				        while (m.find()) {
				        	ananyticAccount = m.group();
				        }
					}
				}
			}
			begin = content.indexOf("google_ad_client = \"");
			String adsAccount = "";
			if(begin>0)
			{
				begin+="google_ad_client = \"', '".length();
				int end =  content.indexOf("\"",begin);
				adsAccount = content.substring(begin, end);
				System.out.println("adsAccount :"+adsAccount);
			}
			
			//Get email
			StringBuffer email = new StringBuffer();
			try
			 {
				final String RE_MAIL = "([\\w\\-]([\\.\\w])+[\\w]+@([\\w\\-]+\\.)+[A-Za-z]{2,4})";
			    Pattern p = Pattern.compile(RE_MAIL);
			    Matcher m = p.matcher(content.replaceAll("\\<.*?\\>", " "));
			    while(m.find()) {
			    	if(m.group(1).endsWith(domainName))
			    	{
			    		if (email.length() > 0)
			    			email.append(",");
			    		email.append(m.group(1).toLowerCase());
			    		
			    	}
			   } 
			 }
			 catch (Exception ex)
			 {
				 ex.printStackTrace();
			 }
			 
			// JS
			List<String> jsLib = new ArrayList<String>();
			if(content.indexOf("jquery")>0)
				jsLib.add("jquery");
			
			if(content.indexOf("prototype.js")>0)
				jsLib.add("prototype");
			
			if(content.indexOf("extjs")>0)
				jsLib.add("extjs");
			
			if(content.indexOf("yui")>0)
				jsLib.add("yui");
			
			if(content.indexOf("dojo")>0)
				jsLib.add("dojo");
			
			if(content.indexOf("gwt")>0)
				jsLib.add("gwt");

			if(content.indexOf("easeljs")>0)
				jsLib.add("easeljs");
			
			//Get list link
			Elements link = doc.select("a");
			
			ArrayList<String> listLink = new ArrayList<String>();
			String check = null;
			for(int i=0;i<link.size();i++)
			{
				check = link.get(i).attr("rel"); 
				if(Pattern.matches("^http[\\w\\W]+", link.get(i).attr("href")))
				{
					if( check != null && !check.equals("nofollow") || check == null)
					{
						Pattern p = Pattern.compile("^http[^\\/]+[\\/]{2}[^\\/]+");
	
				        Matcher m = p.matcher(link.get(i).attr("href"));
				        while (m.find()) {
				        	if(!listLink.contains(m.group()))
				        		listLink.add(m.group());				        	
				        }
					}
				}
			}
			//Get outLink
			List<String> outLink = new ArrayList<String>();
    		if(listLink.size()>0)
    		{
    			for(int i=0;i<listLink.size();i++)
    			{
    				if(!Pattern.matches("(http|https)+://(www.)*" + domainName, listLink.get(i)))
    				{
    					outLink.add(listLink.get(i));
    				}
    			}
	        }
    		/*//Get inLink
    		List<String> inLink = new ArrayList<String>();
    		if(listLink.size()>0)
    		{
    			for(int i=0;i<listLink.size();i++)
    			{
    				if(!Pattern.matches("(http|https)+://(www.)*" + domainName, listLink.get(i)))
    				{
		        		inLink.add("http://" + domainName);
    				}
		        }
    		}*/
    		
    		//Set random
    		Random rand = new Random();
			domain.setTitle(title);
			domain.setKeywords(keywords);
			domain.setDescription(description);
			domain.setGoogleAnalytic(ananyticAccount);
			domain.setGoogleAdsense(adsAccount);
			domain.setProgrammingLanguage(powerBy);
			domain.setWebServer(server);
			domain.setEmail(email.toString());
			domain.setJavascriptLibrary(jsLib);
            domain.setUpdated(new Date());
            domain.setRandom(rand.nextInt(50000000));
            
            if (outLink.size() > 0) {
            	new UpdateOutlink(domainName, outLink);
            }
            
        } catch (MalformedURLException e) {
            e.printStackTrace();
        } catch (IOException e) {
        	e.printStackTrace();
        }
        
        return domain;
	}
	
	public static Domain getIp(Domain domain) {
		String url = "http://www.nearby.org.uk/api/dig.php?l=" + domain.getName();
		try {
			Document doc = Jsoup.connect(url).get();
			String ip = doc.text();
			domain.setIpAddress(ip);
		} catch (Exception e) {
			e.printStackTrace();
		}
		return domain;
	}
	
	public static Domain getAlexa(Domain domain) {
		String url = "http://www.alexa.com/siteinfo/" + domain.getName();
		try {
			Document doc = Jsoup.connect(url).get();
			String content = doc.text();
			
			// visitor
			int begin = content.indexOf("Its visitors view an average of ");
			int visit = 0;
			if(begin>0)
			{
				begin += "Its visitors view an average of ".length();
				int end=content.indexOf(" ",begin);
				String unique = content.substring(begin, end);
				visit = (int) Math.round(Double.parseDouble(unique));
				
			}
			
			// rank
			String ranks = doc.select("#siteStats").select("tbody tr td div").first().text().trim().replace(",","");
			int rank = 0;
			try {
				if(ranks != null && !ranks.equals(""))
				{
					rank=Integer.parseInt( ranks );
				}
			} catch (Exception e) {
				e.printStackTrace();
			}
			
			//keywords
			ArrayList<String> keywords = new ArrayList<String> ();
			Elements elements = doc.select("#top-keywords-from-search");
			
			if(elements.size() > 0)
			{
				for (Element element : elements.first().select("u")) {
					keywords.add(element.text());
				}
			}			

			// category
			List<String> categories = null;
			try {
				elements = doc.select("#siteSentences div").select("a");
				for (Element element : elements) {
					if(categories.size()>5) break; 
					
					String categoryName = element.text();
					categories.add(categoryName);
				}
			} catch (Exception e) {
				// TODO: handle exception
			}
			
			//Visitor by country
			List<String> visitorCountryName = new ArrayList<String>();
			List<String> visitorCountryPercent = new ArrayList<String>();
			Elements countries = doc.select("#visitors-by-country .tr1");
			for (Element country : countries) {
				try {
					visitorCountryName.add(country.select(".tc1").get(0).select("a").html().replaceAll("<[^>]+/>", "").replace("&nbsp;", "").trim());
					visitorCountryPercent.add(country.select(".tc1").get(1).html().replace("%", ""));
				} catch (Exception e) {
					// TODO: handle exception
				}
			}
			
//			domain.setKeywords(StringHelper.arrayListToString(keywords, ","));
			domain.setCategory(categories);
			domain.setVisitorPerDay(visit);
			domain.setAlexaGlobeRank(rank);
			domain.setVisitorCountryName(visitorCountryName);
			domain.setVisitorCountryPercent(visitorCountryPercent);
			
		} catch (Exception e) {
			e.printStackTrace();
		}
		return domain;
	}
	
	public static Domain getGooglePageRank(Domain domain) {
		String message = null;
		try {
			message = URLEncoder.encode(domain.getName(), "UTF-8");
		} catch (UnsupportedEncodingException e1) {
			e1.printStackTrace();
		}

		try {
			String url = "http://gpagerank.net/?url=" + message;
        	Document doc = Jsoup.connect(url).get();
		
        	int googlePageRank = 0;
        	String google = doc.select(".show").select("span").first().select("font").text().replaceAll("[^0-9]", "");
        	if(!google.equals(""))
        	{
        		googlePageRank = Integer.parseInt(google);
        	}
        	domain.setGooglePageRank(googlePageRank);
		} catch (Exception e) {
			e.printStackTrace();
		}
		
        return domain;
	}
	
	public static Domain getAgeDomain(Domain domain) {
		String message = null;
		try {
			message = URLEncoder.encode(domain.getName(), "UTF-8");
		} catch (UnsupportedEncodingException e1) {
			e1.printStackTrace();
		}
		
		URL url;
		try {
			url = new URL("http://www.seologs.com/dns/domain-check.html");
			HttpURLConnection connection = (HttpURLConnection) url.openConnection();
	        connection.setDoOutput(true);
	        connection.setRequestMethod("POST");
	        connection.setRequestProperty("Content-Type", "application/x-www-form-urlencoded"); 
	        OutputStreamWriter writer = new OutputStreamWriter(connection.getOutputStream());
	        writer.write("query=" + message);
	        writer.write("&output=cdt");
	        writer.close();

	        if (connection.getResponseCode() == HttpURLConnection.HTTP_OK) {
	        	BufferedReader reader = new BufferedReader(new InputStreamReader(connection.getInputStream()));
	           	String content = "";
	           	while(1==1)
	    		{
	    			String str = reader.readLine();
	    			if(str==null) break;
	    			content+=str;   				
	    		}
	           	Document doc = Jsoup.parse(content);
	           	String age = doc.select("div[class=KonaBody]").select("p").text();
	           	String[] ageSplit = age.split("(approximately)");
	           	if(ageSplit.length ==2)
	           	{
	           		domain.setAge(ageSplit[1].replaceAll("[):]+", ""));
	           	 }
	        }
	        
		} catch (MalformedURLException e) {
			e.printStackTrace();
		} catch (IOException e) {
			e.printStackTrace();
		}
		return domain;
	}
	
	public static Domain getWhois(Domain domain) {
		try {
			String url = "http://www.whoismind.com/whois/"+domain.getName() + ".html";
			Document doc = Jsoup.connect(url).get();
			
			try {
				String str = doc.select("#whois-rawinfo .whois_raw > div:eq(1)").html();
				domain.setWhois(str);

			} catch (Exception e) {
				// TODO: handle exception
			}
		} catch (Exception e) {
			
			e.printStackTrace();
		}
		
		return domain;
	}
	
	public static Domain getQuantcast(Domain domain) {
		try {
			String url = "http://www.quantcast.com/" + domain.getName();
			Document doc = Jsoup.connect(url).get();
			try {
				String str = doc.select("#PEOPLE-tab").select(".data").html();
				System.out.println(doc.select("#PEOPLE-tab"));
			} catch (Exception e) {
				// TODO: handle exception
			}
			
		} catch (Exception e) {
			e.printStackTrace();
		}
		return domain;
	}
	
	public static Domain getCompete(Domain domain) {
		try {
			String url = "http://siteanalytics.compete.com/" + domain.getName();
			Document doc = Jsoup.connect(url).get();
			try {
				String str = doc.select("body").select(".wrapper .sidebar .score").select("h4").first().html().replace(",", "");
				domain.setAnalysisVisitorsCompete(Integer.parseInt(str));
			} catch (Exception e) {
				// TODO: handle exception
			}
		} catch (Exception e) {
			// TODO: handle exception
		}
		return domain;
	}
	
	public static Domain getRandomSites(Domain domain) {
		//Get 5 domain near site
		int standarRank = 500000;
		//Save data
		try {
			ObjectifyService.register(Domain.class);
		} catch (Exception e) {
			// TODO: handle exception
		}
		List<String> randomSites = new ArrayList<String>();
		Random rand = new Random();
		
		try {
			Objectify ofy = ObjectifyService.begin();
			Query<Domain> q = ofy.query(Domain.class).filter("alexaGlobeRank < ", rand.nextInt(28000000)).filter("alexaGlobeRank !=", 0).order("-alexaGlobeRank").limit(10);
			
			for (Domain d: q) {
				
				randomSites.add(d.getName());
			}
		} catch (Exception e) {
			// TODO: handle exception
			e.printStackTrace();
		}
		
		domain.setRandomSites(randomSites);
		
		return domain;
	}
	
	public static Domain getNearSites(Domain domain) {
		//Save data
		try {
			ObjectifyService.register(Domain.class);
		} catch (Exception e) {
			// TODO: handle exception
		}
		
		List<String> nearSites = new ArrayList<String>();
		Objectify ofy = ObjectifyService.begin();
		
		if (domain.getAlexaGlobeRank() > 0) {
			try {
				//Get 5 site has alexa rank greater this domain
				Query<Domain> q = ofy.query(Domain.class).filter("alexaGlobeRank <", domain.getAlexaGlobeRank()).order("-alexaGlobeRank").limit(5);
				for (Domain d: q) {
					nearSites.add(d.getName());
				}
			} catch (Exception e) {
				// TODO: handle exception
				e.printStackTrace();
			}
			
			try {
				//Get 5 site has alexa rank smaller this domain
				Query<Domain> q2 = ofy.query(Domain.class).filter("alexaGlobeRank >", domain.getAlexaGlobeRank()).filter("alexaGlobeRank !=", 0).order("alexaGlobeRank").limit(5);
				for (Domain d: q2) {
					nearSites.add(d.getName());
				}
			} catch (Exception e) {
				// TODO: handle exception
				e.printStackTrace();
			}
		} else {
			Random rand = new Random();
			Query<Domain> q = ofy.query(Domain.class).filter("alexaGlobeRank < ", rand.nextInt(5000000)).filter("alexaGlobeRank !=", 0).order("-alexaGlobeRank").limit(10);
			for (Domain d: q) {
				nearSites.add(d.getName());
			}
		}
		
		domain.setNearSites(nearSites);
		
		return domain;
	}
}